{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "private_outputs": true,
      "provenance": [],
      "authorship_tag": "ABX9TyNUV1Q81PMrp8t8KqbhzJpI",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ktynski/Marketing_Automations_Notebooks_With_GPT/blob/main/Copy_of_The_Ultimate_AI_Researcher_(Public).ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-_V-mcbY2O1m"
      },
      "outputs": [],
      "source": [
        "!git clone https://github.com/ading2210/poe-api.git"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip3 install poe-api\n",
        "!pip install pdfx\n",
        "!pip install PyPDF2\n",
        "!pip install arxiv\n",
        "!pip install transformers\n",
        "!pip install openai"
      ],
      "metadata": {
        "id": "qLOx2EYj30S5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import urllib.request\n",
        "import poe\n",
        "import json\n",
        "import PyPDF2\n",
        "import pandas as pd\n",
        "import openai\n",
        "import arxiv\n",
        "import concurrent.futures\n",
        "from concurrent.futures import ThreadPoolExecutor, as_completed, wait, FIRST_COMPLETED\n",
        "from transformers import GPT2Tokenizer\n",
        "import time\n",
        "\n",
        "# Define your OpenAI GPT-3 API key\n",
        "openai.api_key = \"your API key\"\n",
        "\n",
        "# Function to convert PDF to text\n",
        "def convert_pdf_to_text(pdf_path):\n",
        "    text = ''\n",
        "\n",
        "    with open(pdf_path, 'rb') as file:\n",
        "        reader = PyPDF2.PdfReader(file)\n",
        "\n",
        "        for page in reader.pages:\n",
        "            text += page.extract_text()\n",
        "\n",
        "    return text\n",
        "\n",
        "\n",
        "def truncate_string_to_tokens(content, max_tokens):\n",
        "    tokenizer = GPT2Tokenizer.from_pretrained(\"gpt2\")\n",
        "    content_tokens = tokenizer.tokenize(content)\n",
        "\n",
        "    if len(content_tokens) > max_tokens:\n",
        "        content_tokens = content_tokens[:max_tokens]\n",
        "\n",
        "    truncated_content = tokenizer.convert_tokens_to_string(content_tokens)\n",
        "    print(\"Truncated text\")\n",
        "    truncated_content = str(truncated_content)\n",
        "    print(truncated_content)\n",
        "    return truncated_content\n",
        "\n",
        "\n",
        "# Function to call GPT-3 and get a response\n",
        "def gpt4_response(prompt):\n",
        "    response = openai.ChatCompletion.create(\n",
        "                model=\"gpt-3.5-turbo\",\n",
        "                messages=[\n",
        "                    {\n",
        "                        \"role\": \"system\",\n",
        "                        \"content\": \"Please rate the relevance of the abstract to the query on a scale of 0 to 100 where 100 is extremely relevant for the query in relation to the paper title.\"\n",
        "                    },\n",
        "                    {\n",
        "                        \"role\": \"user\",\n",
        "                        \"content\": f\"{prompt} \\n Score:\"\n",
        "                    }\n",
        "                ],\n",
        "                max_tokens=1000,\n",
        "                n=1,\n",
        "                stop=None,\n",
        "                temperature=0.7,\n",
        "            )\n",
        "\n",
        "    result = response[\"choices\"][0][\"message\"][\"content\"].strip()\n",
        "    return result\n",
        "\n",
        "\n",
        "# Function to get relevance score using GPT model\n",
        "def get_relevance_score(query, title):\n",
        "    # Create the prompt for GPT\n",
        "    prompt = f\"How relevant is the paper titled '{title}' to the query '{query}'? Please rate on a scale from 0 to 100, with 100 being extremely relevant.\"\n",
        "\n",
        "    # Get the response from GPT\n",
        "    response = gpt4_response(prompt)\n",
        "\n",
        "    # Split the response string into components and get the first one (the score)\n",
        "    score_str = response.split(\".\")[0]\n",
        "\n",
        "    # Convert the score string to a float relevance score\n",
        "    relevance_score = float(score_str)\n",
        "    print(f\"Relevance Score:{relevance_score}\")\n",
        "    return relevance_score\n",
        "\n",
        "\n",
        "# Function to download ArXiv papers\n",
        "def download_arxiv_pdfs(df, download_dir):\n",
        "    for index, row in df.iterrows():\n",
        "        pdf_url = row['pdf_url']\n",
        "        pdf_title = row['title']\n",
        "\n",
        "        # Generate a valid filename by removing invalid characters\n",
        "        pdf_filename = ''.join(c if c.isalnum() else '_' for c in pdf_title)\n",
        "        pdf_path = os.path.join(download_dir, f\"{pdf_filename}.pdf\")\n",
        "\n",
        "        try:\n",
        "            urllib.request.urlretrieve(pdf_url, pdf_path)\n",
        "            print(f\"Downloaded {pdf_title} to {pdf_path}\")\n",
        "        except Exception as e:\n",
        "            print(f\"Failed to download {pdf_title}: {e}\")\n",
        "\n",
        "\n",
        "# Function to get summaries using AI model\n",
        "def get_summaries(client, text):\n",
        "    text = truncate_string_to_tokens(text, 25000)\n",
        "    summaries = []\n",
        "    for chunk in client.send_message(\"a2_100k\", text):\n",
        "        summaries.append(chunk[\"text_new\"])\n",
        "    return ' '.join(summaries)\n",
        "\n",
        "def get_summaries_with_backoff_and_timeout(client, text, retries=5, backoff_factor=0.1, timeout=30):\n",
        "    for i in range(retries):\n",
        "        try:\n",
        "            with ThreadPoolExecutor() as executor:\n",
        "                future = executor.submit(get_summaries, client, text)\n",
        "                done, _ = wait([future], timeout=timeout, return_when=FIRST_COMPLETED)\n",
        "\n",
        "                if future in done:\n",
        "                    return future.result()\n",
        "                else:\n",
        "                    raise TimeoutError(\"The get_summaries function timed out\")\n",
        "\n",
        "        except Exception as e:\n",
        "            if i < retries - 1:  # if not the last attempt\n",
        "                sleep_time = backoff_factor * (2 ** i)  # exponential backoff\n",
        "                time.sleep(sleep_time)\n",
        "            else:\n",
        "                raise e\n",
        "\n",
        "\n",
        "\n",
        "def get_relevance_score_with_backoff_and_timeout(query, title, retries=5, backoff_factor=0.1, timeout=30):\n",
        "    for i in range(retries):\n",
        "        try:\n",
        "            with ThreadPoolExecutor() as executor:\n",
        "                future = executor.submit(get_relevance_score, query, title)\n",
        "                done, _ = wait([future], timeout=timeout, return_when=FIRST_COMPLETED)\n",
        "\n",
        "                if future in done:\n",
        "                    return future.result()\n",
        "                else:\n",
        "                    raise TimeoutError(\"The get_relevance_score function timed out\")\n",
        "\n",
        "        except Exception as e:\n",
        "            if i < retries - 1:  # if not the last attempt\n",
        "                sleep_time = backoff_factor * (2 ** i)  # exponential backoff\n",
        "                time.sleep(sleep_time)\n",
        "            else:\n",
        "                raise e\n",
        "\n",
        "\n",
        "\n",
        "def main(query=\"transgender etiology\", max_results=10, threshold=70, download_dir='/content', sort_by=arxiv.SortCriterion.SubmittedDate):\n",
        "    # Get ArXiv papers and create DataFrame\n",
        "    search = arxiv.Search(\n",
        "        query=str(\"abs:\" + query),\n",
        "        max_results=max_results,\n",
        "        sort_by=sort_by\n",
        "    )\n",
        "\n",
        "    papers = []\n",
        "    for result in search.results():\n",
        "        papers.append(result)\n",
        "    print(papers)\n",
        "    papers_df = pd.DataFrame([vars(paper) for paper in papers])\n",
        "\n",
        "    # Set up POE client. You can get this by logging in to your account in the web browser, right click, go to inspect, then applications, then get the token from the cookie.\n",
        "    client = poe.Client(\"your poe token\")\n",
        "\n",
        "    # Add relevance scores to DataFrame\n",
        "    with ThreadPoolExecutor(max_workers=5) as executor:\n",
        "        future_to_index = {executor.submit(get_relevance_score_with_backoff_and_timeout, query, row['summary']): index for index, row in papers_df.iterrows()}\n",
        "        for future in as_completed(future_to_index):\n",
        "            index = future_to_index[future]\n",
        "            try:\n",
        "                relevance_score = future.result()\n",
        "            except Exception as e:\n",
        "                print(f\"Failed to get relevance score: {e}\")\n",
        "            else:\n",
        "                papers_df.at[index, 'relevance_score'] = relevance_score\n",
        "\n",
        "    # Filter DataFrame to only include papers with a relevance score above the threshold\n",
        "    papers_df = papers_df[papers_df['relevance_score'] > threshold]\n",
        "\n",
        "    # Download PDFs and get summaries for each paper\n",
        "    for index, row in papers_df.iterrows():\n",
        "        pdf_url = row['pdf_url']\n",
        "        pdf_title = row['title']\n",
        "\n",
        "        # Generate a valid filename by removing invalid characters\n",
        "        pdf_filename = ''.join(c if c.isalnum() else '_' for c in pdf_title)\n",
        "        pdf_path = os.path.join(download_dir, f\"{pdf_filename}.pdf\")\n",
        "\n",
        "        try:\n",
        "            urllib.request.urlretrieve(pdf_url, pdf_path)\n",
        "            print(f\"Downloaded {pdf_title} to {pdf_path}\")\n",
        "\n",
        "            text = convert_pdf_to_text(pdf_path)\n",
        "            summary = get_summaries_with_backoff_and_timeout(client, text)\n",
        "\n",
        "\n",
        "            papers_df.at[index, 'poe summary'] = summary\n",
        "\n",
        "            print(f\"Generated summary for {pdf_title}\")\n",
        "        except Exception as e:\n",
        "            print(f\"Failed to process {pdf_title}: {e}\")\n",
        "\n",
        "    # Save the final DataFrame to a CSV file\n",
        "    papers_df.to_csv('papers_with_summaries.csv')\n",
        "    return papers_df\n",
        "\n",
        "# Run the main function\n",
        "if __name__ == \"__main__\":\n",
        "    main(query=\"F5 Tornado\", max_results=50, threshold=50, download_dir='/content')\n",
        "\n"
      ],
      "metadata": {
        "id": "ET8PaRs6ZcQH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "papers_df"
      ],
      "metadata": {
        "id": "Am2RHmFXLJwp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "papers_df.to_csv(\"summaries.csv\")"
      ],
      "metadata": {
        "id": "neR9RWl1acrn"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}
